{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mSAB-uiGZ_0p"
      },
      "source": [
        "***Copyright 2020 Google LLC.***\n",
        "\n",
        "Licensed under the Apache License, Version 2.0 (the \"License\");"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Gexm6jCf1ZpE"
      },
      "source": [
        "**Author:** Furkan Kocayusufoglu \\\\\n",
        "**Term:** Summer 2020 Research Internship with Mixel/Brain \\\\\n",
        "**Purpose:** This notebook processes raw data from amazon into \u003cuser, item, query, time\u003e quadruplets for personalized retrieval task.  \n",
        "**Notes:** Datasets are downloaded from http://deepyeti.ucsd.edu/jianmo/amazon/index.html  "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "iRwykf0ZaAR0"
      },
      "outputs": [],
      "source": [
        "import tensorflow.compat.v1 as tf\n",
        "from collections import defaultdict\n",
        "from datetime import datetime\n",
        "import json\n",
        "import string\n",
        "import os"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7-Z1tXBZ1cj9"
      },
      "outputs": [],
      "source": [
        "# *** Helper functions for raw data stream.\n",
        "# Reads a json file with multiple objects line by line in a streaming fashion. \n",
        "# Since some files are very large, we can read only a specified number of lines (max 100M lines allowed).\n",
        "def parse_stream(path, max_lines=100000000):\n",
        "  with tf.gfile.Open(path, 'r') as f:\n",
        "    for line_num, line in enumerate(f):\n",
        "      if line_num \u003e max_lines:\n",
        "        break\n",
        "      obj = json.loads(line)\n",
        "      yield obj\n",
        "  print(\"Finished reading.\")\n",
        "\n",
        "# Reads a json file with multiple objects, stores them as a list and returns.\n",
        "def parse(path):\n",
        "  data = []\n",
        "  with tf.gfile.Open(path, 'r') as f:\n",
        "    for line in f.readlines():\n",
        "      data.append(json.loads(line))\n",
        "  print(\"Finished reading.\")\n",
        "  return data\n",
        "\n",
        "# *** Helper function for text (query) processing.\n",
        "# Removes duplicate words and punctuations. Duplicate words are removed from the\n",
        "# left to right. E.g., \"Beverages Coffee, Tea \u0026 Cocoa, Tea Samplers\" becomes \n",
        "# \"Beverages Coffee Cocoa Tea Samplers\".\n",
        "def remove_duplicate_words(text):\n",
        "  import string\n",
        "  tokens = text.translate(str.maketrans('', '', string.punctuation)).split()\n",
        "  # tokens = text.split(' ')\n",
        "  ulist = []\n",
        "  for x in reversed(tokens): \n",
        "    if x not in ulist: \n",
        "      ulist.append(x)\n",
        "  return ' '.join(reversed(ulist))\n",
        "\n",
        "  # *** Helper function for joining file paths. Currently not used.\n",
        "  def join_paths(path1, path2):\n",
        "    return os.path.join(path1, path2)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Ukfmq8iP1jRq"
      },
      "outputs": [],
      "source": [
        "# Global variables and paths\n",
        "amazon_data_path = 'path/to/your/data'\n",
        "categories = [\n",
        "  'Sports_and_Outdoors', \n",
        "  'Kindle_Store',\n",
        "  'CDs_and_Vinyl',\n",
        "  'Grocery_and_Gourmet_Food', \n",
        "  'Movies_and_TV',\n",
        "  'Video_Games',\n",
        "  'Pet_Supplies',\n",
        "  'Luxury_Beauty',\n",
        "  'Electronics'\n",
        "]\n",
        "\n",
        "# Users and items with total number of interactions less than this will be ignored.\n",
        "min_interaction_threshold = 5\n",
        "\n",
        "# Data older than this timestamp will be ignored.\n",
        "time_threshold = 1356998400 # Corresponds to Tue 01 Jan 2013 12:00:00 AM UTC"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "OfVG2KNe1rjd"
      },
      "outputs": [],
      "source": [
        "# For each category: \n",
        "# 1. Read the reviews data.\n",
        "# 2. Filter users and items with less than five interactions.\n",
        "# 3. Extract synthetic queries for each item using the category information from metadata.\n",
        "# 4. Map users items and queries to an integer identifier, starting from 1.\n",
        "# 5. Store \u003cuser_id, item_id, query_id, time\u003e quadruplets in file (sorted by time).\n",
        "for category_name in categories:\n",
        "\n",
        "  user_interaction_count = defaultdict(lambda: 0)\n",
        "  item_interaction_count = defaultdict(lambda: 0)\n",
        "  line = 0\n",
        "  largest_time = 0\n",
        "  print(\"*** Category: \", category_name)\n",
        "\n",
        "  f = tf.gfile.Open('{}{}_user_item_time.txt'.format(amazon_data_path, category_name), 'w')\n",
        "  for obj in parse_stream('{}{}.json'.format(amazon_data_path, category_name)):\n",
        "    time = obj['unixReviewTime']\n",
        "    if time \u003c time_threshold:\n",
        "      continue\n",
        "    user = obj['reviewerID']\n",
        "    item = obj['asin']  \n",
        "    largest_time = max(time, largest_time)\n",
        "    user_interaction_count[user] += 1\n",
        "    item_interaction_count[item] += 1\n",
        "\n",
        "    # Writing \u003cuser item time\u003e triplets per line.\n",
        "    f.write(\" \".join([user, item, str(time)]) + ' \\n')\n",
        "    line += 1\n",
        "    if line  == 1:\n",
        "      print(\"Started writing...\")\n",
        "    if line % 100000 == 0:\n",
        "      print(\"At line: \", line, \" with largest time so far: \", largest_time)\n",
        "  f.close()\n",
        "\n",
        "  user_map = dict()\n",
        "  user_num = 0\n",
        "  item_map = dict()\n",
        "  item_num = 0\n",
        "  user_item_seq = dict()\n",
        "  count_lt5 = 0\n",
        "\n",
        "  with tf.gfile.Open('{}{}_user_item_time.txt'.format(amazon_data_path, category_name), 'r') as f:\n",
        "    for line in f:\n",
        "      user, item, time = line.rstrip().split(' ')\n",
        "      if min(user_interaction_count[user], item_interaction_count[item]) \u003c min_interaction_threshold:\n",
        "        count_lt5 += 1\n",
        "        continue\n",
        "\n",
        "      if user in user_map:\n",
        "        user_id = user_map[user]\n",
        "      else:\n",
        "        user_num += 1\n",
        "        user_id = user_num\n",
        "        user_map[user] = user_id\n",
        "        user_item_seq[user_id] = []\n",
        "      if item in item_map:\n",
        "        item_id = item_map[item]\n",
        "      else:\n",
        "        item_num += 1\n",
        "        item_id = item_num\n",
        "        item_map[item] = item_id\n",
        "      \n",
        "      user_item_seq[user_id].append([item_id, time])\n",
        "\n",
        "  print(\"Total number of users with \u003e= 5 interactions: \", user_num)\n",
        "  print(\"Total number of items with \u003e= 5 interactions: \", item_num)\n",
        "  print(\"Number of times we skipped an item (or user): \", count_lt5)\n",
        "\n",
        "  # Sort reviews in user_item_seq according to time\n",
        "  for user_id in user_item_seq.keys():\n",
        "    user_item_seq[user_id].sort(key=lambda x: x[1])\n",
        "\n",
        "  # Extract queries for each item.\n",
        "  item_query_id_map = dict()\n",
        "  query_num = 0\n",
        "  query_id_map = dict()\n",
        "  id_query_map = dict()\n",
        "\n",
        "  # Keep a special query for items without category information.\n",
        "  no_category_query = 'NOC'\n",
        "  query_num += 1\n",
        "  query_id_map[no_category_query] = query_num\n",
        "  id_query_map[query_num] = no_category_query\n",
        "\n",
        "  line = 0\n",
        "  for obj in parse_stream('{}meta_{}.json'.format(amazon_data_path, category_name)):\n",
        "    if 'category' not in obj:\n",
        "      query = no_category_query\n",
        "    else: \n",
        "      query = \" \".join(obj['category'])\n",
        "      query = remove_duplicate_words(query)\n",
        "    \n",
        "    item = obj['asin']\n",
        "    # Item needs to be in the selected group.\n",
        "    if item not in item_map:\n",
        "      continue\n",
        "    \n",
        "    line += 1\n",
        "    if line == 1:\n",
        "      print(\"Started reading metadata...\")\n",
        "    if line % 100000 == 0:\n",
        "      print(\"At line: \", line, len(query_id_map), len(id_query_map)) # Last two should match\n",
        "    \n",
        "    if query in query_id_map:\n",
        "      query_id = query_id_map[query]\n",
        "    else:\n",
        "      query_num += 1\n",
        "      query_id = query_num\n",
        "      query_id_map[query] = query_id\n",
        "      id_query_map[query_id] = query\n",
        "    \n",
        "    item_query_id_map[item_map[item]] = query_id\n",
        "\n",
        "  # Generate an output file with \u003cuser_id item_id query_id time\u003e per line. \n",
        "  f = tf.gfile.Open('{}{}_user_item_query_time_mapped.txt'.format(amazon_data_path, category_name), 'w')\n",
        "  for user in user_item_seq.keys():\n",
        "    for i in user_item_seq[user]:\n",
        "      # i: [item_id, time] -\u003e query: item_query_id_map[i[0]]\n",
        "      \n",
        "      # A safety check in case any of items aren't associated with a query. This\n",
        "      # could happen if metadata is missing an item.\n",
        "      if i[0] not in item_query_id_map:\n",
        "        continue\n",
        "      f.write('%d %d %d %s\\n' % (user, i[0], item_query_id_map[i[0]], str(i[1])))\n",
        "  f.close()\n",
        "\n",
        "  # Store item, user and query mapping dicts for interpretability/debug purposes.\n",
        "  # Maps have the following formats:\n",
        "  # User -\u003e user: user_id\n",
        "  # Item -\u003e item: item_id\n",
        "  # Query -\u003e query_id: query\n",
        "  with tf.gfile.Open('{}{}_itemmap.json'.format(amazon_data_path, category_name), 'w') as f:\n",
        "    json.dump(item_map, f)\n",
        "  with tf.gfile.Open('{}{}_usermap.json'.format(amazon_data_path, category_name), 'w') as f:\n",
        "    json.dump(user_map, f)\n",
        "  with tf.gfile.Open('{}{}_querymap.json'.format(amazon_data_path, category_name), 'w') as f:\n",
        "    json.dump(id_query_map, f)\n",
        "  \n",
        "  print(\"Finished category: \", category_name)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "bKZH9kpa7l2c"
      },
      "outputs": [],
      "source": [
        ""
      ]
    }
  ],
  "metadata": {
    "colab": {
      "collapsed_sections": [],
      "last_runtime": {
        "build_target": "//learning/deepmind/public/tools/ml_python:ml_notebook",
        "kind": "private"
      },
      "name": "data_preprocessing.ipynb",
      "provenance": [
        {
          "file_id": "18K6BDo3T5WB43EKmXshCrADqH5ZJ84ph",
          "timestamp": 1600985697816
        }
      ]
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}
